package org.wj.arithmetic

import org.apache.commons.lang3.StringUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable.ListBuffer

object MapPartitions extends App {


  val conf = new SparkConf().setMaster("local").setAppName(this.getClass.getName)

  val sc = new SparkContext(conf)

  private val rdd: RDD[String] = sc.textFile("P:\\Project\\Idea\\bigdata\\spark\\src\\main\\resources\\data\\scores.txt")

  rdd.filter(StringUtils.isNotBlank(_)).map(line=>{
    val lineSplit = line.split(" ")
    (lineSplit(0), lineSplit(1))
  })
    .mapPartitions(iter=>{
      val list = ListBuffer[(String, Double)]()
      while(iter.hasNext) {
        val tuple = iter.next()
        list += ((tuple._1, tuple._2.toDouble))
      }
      list.iterator
    }, preservesPartitioning = true)
    .foreach(println(_))


}
